# codeing:utf-8
# -!- coding: utf-8 -!-
import datetime
from conf import *
import pymysql
import os
import AnalyzeTableMain as AnalyzeTableMain
import AnalyzeTextMain as AnalyzeTextMain
import difflib
from pandas import DataFrame
import re
from loguru import logger
import zipfile
import json
import hashlib
import shutil
import time
from multiprocessing import Process,Queue
from common import *

class xm_to_id():
    def __init__(self):
        self.html = ""

    def get_ToChinaText(self, value):
        v = [['[Ⅰ一１]', '1'], ['[Ⅱ二２]', '2'], ['[Ⅲ三３]', '3'], ['[Ⅳ四４]', '4'], ['[Ⅴ五５]', '5'],
             ['[Ⅵ六６]', '6'], ['[Ⅶ七７]', '7'], ['[Ⅷ八８]', '8'], ['[Ⅸ九９]', '9'], ['十', '10'],
             ['[ａＡ]', 'a'], ['[ｂＢ]', 'b'], ['[ｃＣ]', 'c'], ['[ｄＤ]', 'd'], ['[ｅＥ]', 'e'], ['[ｆＦ]', 'f'], ['[ｇＧ]', 'g'],
             ['[ｈＨ]', 'h'],
             ['[ｉＩ]', 'i'], ['[ｊＪ]', 'j'], ['[ｋＫ]', 'k'], ['[ｌＬ]', 'l'], ['[ｍＭ]', 'm'], ['[ｎＮ]', 'n'], ['[ｏＯ]', 'o'],
             ['[ｐＰ]', 'p'],
             ['[ｑＱ]', 'q'], ['[ｒＲ]', 'r'], ['[ｓＳ]', 's'], ['[ｔＴ]', 't'], ['[ｕＵ]', 'u'], ['[ｖＶ]', 'v'], ['[ｗＷ]', 'w'],
             ['[ｘＸ]', 'x'],
             ['[ｙＹ]', 'y'], ['[ｚＺ]', 'z']]
        for i in v:
            value = re.sub(i[0], i[1], value)
        return value

    def removePunctuation(self, value):
        return re.sub(r'(－|,|，|\.|。|/|•|·|、|\||#|;|；|''|‘|’|“|”|《|》|\<|\>|:|：|？|\?|\[|\]|【|】|\{|\}|｛|｝|_|~|!|！|-|`|@|\$|￥|%|…|\^|&|\*|"|\=|\+|\(|\)|（|）|——|—|～)', "", value, flags=re.S | re.I)

    def get_str_sort(self, value):
        v = list(set(re.sub(r'\s+', '', value, flags=re.S | re.I)))
        v.sort()
        return "".join(v)

    def get_id(self, value):
        v = self.get_ToChinaText(value)
        v = re.sub(r'总价|承包|设计施工一体化|采购|监理|EPC|工程管理|工程咨询|项目管理|项目|施工|勘察|设计|招标|代理|合同验收|标段|第|联合体|总承包|专业承包|标|单位|建设|不分标段|投资', '', v, flags=re.S | re.I)
        v = re.sub(r'([^\d])20\d{6}([^\d])', '\1\2', v, flags=re.S | re.I)
        return self.get_str_sort(self.removePunctuation(v))

    def get_name(self, value):
        return re.sub(r'磋商|性|全电子|联合体|网上|竞价|关于|变更|正常|及|的|[开中招评投]标|结果|公[示告布开]|通知|投资人|成交|候选(人|单位)|表|复核|重新|拟|标准|文本|选择|一览|记录|情况|报告|书|第|[0-9一二三四五六七八九十]+次|[复重]评|项目|代码|资格|预审|竞争|性|谈判|\d{4}年\d{2}月\d{2}日|验收|结果|联合体(\d+)?', '', self.removePunctuation(value).upper(), flags=re.S | re.I)

class Producer(Process):
    def __init__(self,file_list):
        super().__init__()
        self.file_list = file_list
        self.father_path = 'E:/project/DataBaseData/'
        self.father_noanalyze = self.father_path + 'NoAnalyze'

    def run(self):
        type_dir = ['1', '2', '3', '4', '5']
        while True:
            logger.info('----------having a part----------')
            for typee in type_dir:
                date_dir = os.listdir(os.path.join(self.father_noanalyze, typee))
                for data_source_id in date_dir:
                    data_source_id_dir = os.listdir(os.path.join(self.father_noanalyze, typee, data_source_id))
                    for date in data_source_id_dir:
                        zip_dir = os.listdir(os.path.join(self.father_noanalyze, typee,data_source_id,date))
                        for zipp in zip_dir:
                            self.file_list.put([typee,data_source_id,date,zipp])
                    self.file_list.put(['data_source_id end',typee,data_source_id])
            while self.file_list.qsize() != 0:
                time.sleep(20)
            time.sleep(120)

class Analyze(Process):


    def __init__(self,file_list):
        super().__init__()
        self.zippnum = 100
        self.file_list = file_list
        self.father_path = 'E:/project/DataBaseData/'
        self.father_noanalyze = self.father_path + 'NoAnalyze'
        self.father_yesanalyze = self.father_path + 'YesAnalyze'
        self.father_analyzedata = self.father_path + 'AnalyzeData/AnalyzeDataNew'



    def wrong_analyze_insert(self,conn,cursor,id_not_change):
        sql = 'insert into analyze_wrong_data(zip_name,identify,wrong_type,date_flag) values(%s,%s,%s,%s)'
        try:
            conn.ping(reconnect=True)
            cursor.executemany(sql, id_not_change)
            conn.commit()
        except:
            DataFrame([id_not_change]).to_csv('wrong_analyze.csv', header=None, index=None, mode='a')


    def write_io(self,typee,data_source_id,end_result,identify):
        try:
            dir_now = self.father_analyzedata + str(typee)
            if not os.path.exists(os.path.join(dir_now,str(data_source_id))):
                os.mkdir(os.path.join(dir_now,str(data_source_id)))
            namee = os.path.join(dir_now,str(data_source_id),datetime.datetime.strftime(datetime.datetime.now(),'%Y%m%d%H%M%S')+'_'+identify+'.json')
            with open(namee,'w',encoding = 'utf-8') as f:
                f.write(end_result)
            return True,namee
        except Exception as e:
            logger.debug('write wrong '+str(e))
            return False,namee


    def rename_io(self,typee,data_source_id,date,zipp):
        try:
            if os.path.exists(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp)):
                if not os.path.exists(os.path.join(self.father_yesanalyze,typee,data_source_id)):
                    os.mkdir(os.path.join(self.father_yesanalyze,typee,data_source_id))
                if not os.path.exists(os.path.join(self.father_yesanalyze, typee, data_source_id,date)):
                    os.mkdir(os.path.join(self.father_yesanalyze, typee, data_source_id,date))
                os.rename(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp),os.path.join(self.father_yesanalyze,str(typee),data_source_id,date,zipp))
        except Exception as e:
            logger.debug('rename wrong '+str(e))

    def zipe(self,typee,key = False):
        data_ls = os.listdir(os.path.join(self.father_analyzedata+str(typee)))
        for zip_lsi in data_ls:
            if '.zip' in zip_lsi or 'back_up' in zip_lsi:
                continue
            json_data = os.listdir(os.path.join(self.father_analyzedata+str(typee),zip_lsi))
            if len(json_data) >= self.zippnum or key:
                shutil.copyfile(os.path.join(self.father_path, 'TaskEnd'),os.path.join(self.father_analyzedata + str(typee), zip_lsi, 'TaskEnd'))
                try:
                    zip_1 = zipfile.ZipFile(os.path.join(self.father_analyzedata+str(typee),f'{zip_lsi}_{datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d%H%M%S")}.zip'),'w')
                    for data in os.listdir(os.path.join(self.father_analyzedata+str(typee),zip_lsi)):
                        zip_1.write(os.path.join(self.father_analyzedata+str(typee), zip_lsi, data),data,zipfile.ZIP_DEFLATED)
                    zip_1.close()
                    if os.path.exists(os.path.join(self.father_analyzedata+str(typee), zip_lsi)):
                        shutil.rmtree(os.path.join(self.father_analyzedata+str(typee), zip_lsi))
                except Exception as e:
                    logger.debug('zip wrong '+str(e))


    def concat(self,end_result):
        end_dict = {'业绩':{},'招标公告':{}}
        for typee in end_result:
            if typee == '表格':
                if end_result[typee] == {}:
                    end_dict['业绩'] = {}
                    end_dict['招标公告'] = {}
                    continue
                end_dict['业绩'] = end_result[typee]['yeji']
                end_dict['招标公告'] = end_result[typee]['zb']
            else:
                if end_dict['业绩'] in [[],['record'],{}]:
                    end_dict['业绩'] = end_result[typee]['yeji']
                else:
                    yeji_ls_concat = ['第一中标候选人','第二中标候选人','第三中标候选人']
                    for yeji_ls_concati in yeji_ls_concat:
                        end_dict['业绩'][yeji_ls_concati] = end_dict['业绩'][yeji_ls_concati] + end_result[typee]['yeji'][yeji_ls_concati]
                    yeji_ls = ['项目名称', '项目类型', '评标成员', '标段']
                    for yeji_lsi in yeji_ls:
                        if end_dict['业绩'][yeji_lsi] == '' and end_result[typee]['yeji'][yeji_lsi] != '':
                            end_dict['业绩'][yeji_lsi] = end_result[typee]['yeji'][yeji_lsi]
                zb_ls = ['项目名称','标的信息','标段', '业主名称', '业主联系人', '业主联系电话', '业主地址', '业主传真',
                   '招标代理单位名称', '招标代理单位联系人', '招标代理单位联系电话', '招标代理单位传真', '招标代理单位地址',
                   '工期', '项目类型', '项目地点', '项目地点城市', '项目地点省份', '建设规模', '招标公告发布日期', '报名截止日期',
                   '投标要求', '发布公告的媒介', '评标成员','招标文件获取的时间','招标文件获取的地点','招标文件获取的方法','招标文件获取的售价','获取招标文件','资金来源',
                    '开标地点','标段划分','招标范围','投资额','采购需求','是否接受联合体','开标地点','发布公告的媒介_来源链接','开标时间','核准文号','采购方式','最高限价',
                   '预算金额','代理服务收费标准及金额','项目编号','核准单位','合同履行期限','标的信息']
                if end_dict['招标公告'] in ([],{}):
                    end_dict['招标公告'] = end_result[typee]['zb']
                else:
                    for zb_lsi in zb_ls:
                        if end_dict['招标公告'][zb_lsi] in ['',[]] and  end_result[typee]['zb'][zb_lsi] not in ['',[]]:
                            end_dict['招标公告'][zb_lsi] = end_result[typee]['zb'][zb_lsi]

        yeji_ls_concat = ['第一中标候选人', '第二中标候选人', '第三中标候选人']
        for yeji_ls_concati in yeji_ls_concat:
            company_dict = {}
            out_result = []
            if end_dict['业绩'] in ([],['record']):
                end_dict['业绩'] = {}
                break
            for i in end_dict['业绩'][yeji_ls_concati]:
                if i['项目负责人_资格_编号'] in ['_','__']:
                    i['项目负责人_资格_编号'] = ''
                key = i['公司名称']
                company_dict[key] = company_dict.get(key, []) + [i]
            for i in company_dict:
                biao_dict = {}
                for j in company_dict[i]:
                    key = j['标段']
                    biao_dict[key] = biao_dict.get(key, []) + [j]
                for j in biao_dict:
                    if len(biao_dict[j]) == 1:
                        out_result.append(biao_dict[j][0])
                    else:
                        biao_out = biao_dict[j][0]
                        for m in biao_dict[j][1:]:
                            try:
                                money = float(m['中标金额'])
                                if money > biao_out['中标金额']:
                                    biao_out = m
                            except:
                                if biao_out['中标金额'] == 0 or biao_out['中标金额'] == '' or biao_out['中标金额'] == '/':
                                    biao_out = m
                        out_result.append(biao_out)
            end_dict['业绩'][yeji_ls_concati] = out_result
        return end_dict


    def handle(self,**kwargs):
        conn = kwargs['conn']
        cursor = kwargs['cursor']
        params = kwargs['params']
        config = kwargs['config']
        json_data = kwargs['json_data']
        date = kwargs['date']
        typee= kwargs['typee']
        zipp = kwargs['zipp']
        json_data_be = kwargs['json_data_be']
        id_not_change = []
        data_type = json_data_be['data_type']
        data_source_id = json_data_be['data_source_id']
        params['data_source_id'] = data_source_id
        title = json_data_be['title']
        title_date = json_data_be['titleDate']
        url = json_data_be['url']
        html = json_data_be['html_content']
        params['html'] = html
        identify = hashlib.md5((title + url + title_date).encode('utf-8')).hexdigest()
        table_name = json_data_be['table_name']
        if data_type in [1,4]:
            html_use = html
        else:
            html_use = ''
        end_result = {"项目名称采集": title, "中标日期": str(title_date), "中标数据": {}, "网页链接": url,"任务ID": data_source_id,'任务表':table_name,'type':data_type,'html':html_use}
        end_dict_be = {'yeji': {'第一中标候选人': [], '第二中标候选人': [], '第三中标候选人': [],'项目名称':'', '项目类型':'', '评标成员':'', '标段':''}, 'zb': {}}
        need_ls = ['项目名称', '标段', '标的信息', '业主名称', '业主联系人', '业主联系电话', '业主地址', '业主传真',
                   '招标代理单位名称', '招标代理单位联系人', '招标代理单位联系电话', '招标代理单位传真', '招标代理单位地址',
                   '工期', '项目类型', '项目地点', '项目地点城市', '项目地点省份', '建设规模', '招标公告发布日期', '报名截止日期',
                   '投标要求', '发布公告的媒介', '评标成员', '招标文件获取的时间', '招标文件获取的地点', '招标文件获取的方法', '招标文件获取的售价',
                   '获取招标文件', '资金来源',
                   '开标地点', '标段划分', '招标范围', '投资额', '采购需求', '是否接受联合体', '开标地点', '发布公告的媒介_来源链接', '开标时间',
                   '核准文号', '采购方式', '最高限价',
                   '预算金额', '代理服务收费标准及金额', '项目编号', '核准单位', '合同履行期限']
        aa = {}
        for aai in need_ls:
            if aai == '投标要求':
                aa[aai] = []
            else:
                aa[aai] = ''
        end_dict_be['zb'] = aa
        end_dict = end_dict_be.copy()
        try:
            read = AnalyzeTableMain.AnalyzeTable(**params)
            end_dict = read.main()
        except Exception as e:
            id_not_change.append((table_name,identify,0,date))
            logger.info(e)
            try:
                DataFrame([[e,url,data_source_id]]).to_csv('wrong_table.csv',header = None,index = None,mode = 'a')
            except Exception as e:
                logger.info('import worng '+ str(e))
        end_result["中标数据"]['表格'] = end_dict
        end_dict =  end_dict_be.copy()
        try:
            read = AnalyzeTextMain.AnalyzeText(**params)
            end_dict = read.main()
        except Exception as e:
            id_not_change.append((table_name, identify,1,date))
            logger.info(str(e))

        end_result["中标数据"]['文本'] = end_dict
        end_result["中标数据"] = self.concat(end_result["中标数据"])
        company_ls = ['第一中标候选人', '第三中标候选人', '第三中标候选人']
        xid = xm_to_id()
        xm_info_ = {"项目标识": "", "项目名称": xid.get_name(end_result['项目名称采集']),
                    '公司名称': '',
                    "标段": "", "中标金额": "", "项目负责人": "",
                    "项目分类": ""}
        company_handle = {}
        if end_result["中标数据"]['业绩'] not in ([], {}):
            for j in company_ls:
                if end_result["中标数据"]['业绩'][j] == []:
                    company_handle[j] = []
                else:
                    for i in end_result["中标数据"]['业绩'][j]:
                        v = re.sub('(司|院|局|队|中心|所)[与和]', r'\1;', i["公司名称"])
                        v = re.sub(
                            "(（?(牵头人|成员|联合体)）?|;|：|◄|、|(设计|施工|监理|勘察)方：?|/|&|名称|第[0-9一二三四五六七八九十]+包|包件[0-9一二三四五六七八九十]+|，|,|[0-9一二三四五六七八九十]+包)|？|；|第一中标(供应商|候选人)|-|／|－|（包\d+）|（包[一二三四五六七八九十]+）|（\d+）|（[a-z](片区|包)）|（包[a-z]）|([中投]标|成交)(报?价|金额|单位|企业|人)为?|\d+(\.\d+)?万?元|(成交)?报价|拟?确定|最低的?",
                            ":", v, flags=re.S | re.I)
                        v = re.sub(u"\u3000", u"", v, flags=re.S | re.I)
                        line = re.split(":", v)
                        for k in line:
                            xm_info = xm_info_.copy()
                            val_obj = re.search(config['company_chose'][0], k, re.S | re.I)
                            if (k != "") and (len(k) >= 5) and val_obj:
                                if k[0] == '（':
                                    k = k[1:]
                                if (k[-1] == '为') or (k[-1] == '）') or (k[-1] == '(') and (
                                        '(' not in k):
                                    k = k[:-1]
                                xm_info["公司名称"] = k
                                xm_info["标段"] = i["标段"] if i["标段"] != '' else \
                                end_result["中标数据"]['业绩']['标段']
                                xm_info["中标金额"] = i["中标金额"]
                                if len(i["项目负责人"]) >= 2:
                                    value = i["项目负责人"]
                                    for r in config['replace_manage_name']:
                                        value = re.sub(r[0], r[1], value, flags=re.S | re.I)
                                    dd = []
                                    d = re.split(":", value)
                                    for di in d:
                                        if di in ['施工', '设计', '勘察', '监理']:
                                            continue
                                        if (len(di) >= 2) and (len(di) <= 4):
                                            obj = re.match(config['human_chose'][0], di,
                                                           re.S | re.I)
                                            if obj:
                                                dd.append(di)
                                    if len(dd) > 0:
                                        xm_info["项目负责人"] = dd[0]
                                if difflib.SequenceMatcher(None, end_result['中标数据']['业绩']["项目名称"],
                                                           end_result[
                                                               '项目名称采集']).quick_ratio() >= 0.8:
                                    if end_result['中标数据']['业绩']["项目名称"] in i["标段"]:
                                        xm_info["项目名称"] = i["标段"]
                                    else:
                                        xm_info["项目名称"] = end_result['中标数据']['业绩']["项目名称"] + i["标段"]
                                else:
                                    if end_result['项目名称采集'] in i["标段"]:
                                        xm_info["项目名称"] = i["标段"]
                                    else:
                                        xm_info["项目名称"] = end_result['项目名称采集'] + i["标段"]
                                if xm_info["项目名称"] == '':
                                    xm_info["项目名称"] = end_result['项目名称采集']
                                xm_info["项目名称"] = xid.get_name(xm_info["项目名称"])
                                xm_info["项目标识"] = xid.get_id(xm_info["项目名称"])
                                company_handle[j] = company_handle.get(j, []) + [xm_info]
            for company_lsi in company_ls:
                if company_lsi in company_handle:
                    end_result["中标数据"]['业绩'][company_lsi] = company_handle[company_lsi]
        else:
            end_result["中标数据"]['业绩'] = {"第一中标候选人": [], "第二中标候选人": [], "第三中标候选人": []}
        end_result = json.dumps(end_result, ensure_ascii=False)
        if id_not_change != []:
            self.wrong_analyze_insert(conn,cursor,id_not_change)
        id_not_change = []
        key,namee = self.write_io(typee,data_source_id,end_result,identify)
        if not key:
            if os.path.exists(namee):
                os.remove(namee)
            DataFrame([[typee,data_source_id,date, zipp, json_data]]).to_csv('writeio_wrong.csv', header=None, index=None, mode='a')


    def run(self):
        while True:
            try:
                conn = pymysql.connect(
                    host='192.168.0.160',
                    port=3306,
                    user='root',
                    passwd='lubanjianye',
                    charset='utf8',
                    database='data_collection_server',
                    cursorclass=pymysql.cursors.DictCursor)
                cursor = conn.cursor()
                args = {'conn': conn,
                        'cursor': cursor}
                conf = Conf(**args)
                config = conf.main()
                confunc = ConFunc(config)
                rubbish_word = confunc.rubbish_word
                replace_word = confunc.replace_word
                get_money = confunc.get_money
                clear_solo = confunc.clear_solo
                params = {
                    'html': 'C:/Users/Administrator/Desktop/table_easy/table30.html',
                    'config': config,
                    'rubbish_word': rubbish_word,
                    'replace_word': replace_word,
                    'get_money': get_money,
                    'database': True,
                    'data_source_id': '',
                    'clear_solo': clear_solo
                }
                break
            except Exception as e:
                logger.debug('pre wrong '+str(e))
                continue
        handle_params = {
            'conn': conn,
            'cursor': cursor,
            'params': params,
            'config': config,
            'date': '',
            'typee': '',
            'zipp': '',
            'json_data_be': '',
            'json_data': ''
        }
        while True:
            all_use_data = self.file_list.get()
            if all_use_data[0] == 'data_source_id end':
                self.zipe(all_use_data[1],True)
                continue
            typee = all_use_data[0]
            data_source_id = all_use_data[1]
            date = all_use_data[2]
            zipp = all_use_data[3]
            logger.info(f'----------{zipp} start----------')
            if not os.path.exists(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp)):
                continue
            try:
                z = zipfile.ZipFile(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp))
            except:
                logger.debug('not a zip {0}'.format(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp)))
                continue
            json_name = z.namelist()
            if typee == 3:
                sql = f'select data_extra from id_to_name_handle where data_source_id = {data_source_id}'
                cursor.execute(sql)
                data_extra = cursor.fetchone()['data_extra']
                if data_extra == 0:
                    self.rename_io(typee,data_source_id,date,zipp)
                    continue
            for json_data in json_name:
                if '任务' in json_data:
                    continue
                try:
                    json_data_be = json.loads(z.read(json_data).decode('utf-8'))
                except Exception as e:
                    logger.debug('loads wrong '+str(e))
                    DataFrame([[typee,data_source_id,date,zipp,json_data]]).to_csv('loads_wrong.csv', header=None, index=None, mode='a')
                    continue
                handle_params['date'] = date
                handle_params['typee'] = typee
                handle_params['zipp'] = zipp
                handle_params['json_data_be'] = json_data_be
                handle_params['json_data'] = json_data
                try:
                    self.handle(**handle_params)
                except Exception as e:
                    logger.debug('handle wrong ' + str(e))
                    DataFrame([[typee, data_source_id,date, zipp, json_data]]).to_csv('handle_wrong.csv', header=None,index=None, mode='a')
                    continue
            z.close()
            self.rename_io(typee,data_source_id,date,zipp)
            self.zipe(typee,False)
            handle_params['json_data_be'] = ''


def main():
    file_list = Queue(100000)
    for i in range(1):
        t = Producer(file_list)
        t.start()
    time.sleep(0.5)
    for i in range(1):
        t = Analyze(file_list)
        t.start()

if __name__ == '__main__':
    main()

